# -*- coding: utf-8 -*-
"""
@Time    : 2024/7/27 17:11 
@Author  : ZhangShenao 
@File    : 3.使用RecursiveCharacterTextSplitter衍生代码分割器.py 
@Desc    : 使用RecursiveCharacterTextSplitter衍生代码分割器

RecursiveCharacterTextSplitter可以创建特定编程语言下的分割器
支持按照特定语言的语法进行文本分割
"""
from langchain_community.document_loaders import UnstructuredFileLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter, Language

# 加载python源文件
loader = UnstructuredFileLoader('./demo.py')
docs = loader.load()

# 创建Python语言的文本分割器
splitter = RecursiveCharacterTextSplitter.from_language(
    language=Language.PYTHON,  # 指定语言模板,每种编程语言有自己的分隔符列表
    chunk_size=500,  # 指定文档分块大小
    chunk_overlap=50,  # 指定文档块之间的重叠部分大小
    add_start_index=True,  # 添加文档块的起始索引
)

# 基于Python的特定语法,切割文档
# 分隔符列表为:["\nclass ","\ndef ","\n\tdef ","\n\n","\n"," ",""]
chunks = splitter.split_documents(docs)

for chunk in chunks:
    print(f'文档块大小: {len(chunk.page_content)}, 元数据: {chunk.metadata}')

print(chunks[2].page_content)
